In [1]:
#library for data manipulation
import pandas as pd
import numpy as np
# for Plotting and visualization
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
In [2]:
# Importing the data
USA = pd.read_csv('E:\\Data for Assignment\\US_Accidents_May19.csv')
Business Problem:
<font color = 'blue'>Background:¶
This is a countrywide traffic accident dataset, which covers 49 states of the United States. The data is collected from February 2016 to March 2019, using several data providers, including two APIs which provide streaming traffic event data. These APIs broadcast traffic events captured by a variety of entities, such as the US and state departments of transportation, law enforcement agencies, traffic cameras and traffic sensors within the road-networks. Currently, there are about 2.25 million accident records in this dataset. </font>
US-Accidents - OBJECTIVE :¶
- To Studying accident hotspot locations
- Casualty analysis
- Extracting cause
- Effect rules to predict accidents
- Studying the impact of precipitation
- Other environmental stimuli on accident occurrence.
In [3]:
# Finding the number of Rows and Columns:
USA.shape
Out[3]:
In [4]:
# To check what headers are there:
USA.head()
Out[4]:
In [5]:
fig=sns.heatmap(USA[['TMC','Severity','Start_Lat','End_Lat','Distance(mi)','Temperature(F)','Wind_Chill(F)','Humidity(%)','Pressure(in)','Visibility(mi)','Wind_Speed(mph)']].corr(),annot=True,cmap='RdYlGn',linewidths=0.2,annot_kws={'size':15})
fig=plt.gcf()
fig.set_size_inches(18,15)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
<font color = 'orange'>Based on the Correlations graph, it show that Start_Lat has good Positive correlation with End_Lat and vice versa. </font>¶
<font color = 'orange'> Temperature has good positive correlation with Wind_chill and vice versa. </font>¶
<font color = 'orange'> Start_Lat & End_Lat has Negative correlation with Temperature and vice versa followed by Wind_Chills.</font>¶
In [6]:
# Checking for the Data types for each of the variable.
USA.info()
In [7]:
# To check the Count, Meanm STd Dev, Max and Min for the dataset.
USA.describe()
Out[7]:
Based on the correlations, below columns are removed as they will not add any value in prediction of the Severity of Accidents in US.¶
EXPLORATORY DATA ANALYSIS
Step – 1:
Removing unnecessary Columns: Removed the columns that will not help in Prediction of Accidents: Below are the Columns that were removed and reasons mentioned:¶
In [8]:
USA = USA.drop(["ID","Source","TMC","Start_Lat","Start_Lng","End_Lat","End_Lng","Distance(mi)","Description",
"Number","Street","Side","City","County","Zipcode","Country","Airport_Code","Weather_Timestamp","Wind_Direction","Precipitation(in)"],axis = 1)
In [9]:
# Reading the names of the columns after removing the columns, just to check if the defined columns are removed.
USA.columns
Out[9]:
In [10]:
# Finding the number of Rows and Columns:
USA.shape
Out[10]:
In [11]:
# Treading start date and time
USA['st_year'] = pd.DatetimeIndex(USA['Start_Time']).year
USA['st_Month'] = pd.DatetimeIndex(USA['Start_Time']).month
USA.head()
Out[11]:
In [12]:
USA[USA.st_year != '2016']
Out[12]:
In [13]:
# Checking the data points for each of the year
USA.groupby('st_year')["st_year"].count()
Out[13]:
In [14]:
# Removing the all the year data points other than 2019, as the dataset size is more, we decided to go with only 2019 data points.
USA.drop(USA[USA['st_year'] == 2016].index, inplace = True)
USA.drop(USA[USA['st_year'] == 2017].index, inplace = True)
USA.drop(USA[USA['st_year'] == 2015].index, inplace = True)
USA.drop(USA[USA['st_year'] == 2018].index, inplace = True)
In [15]:
USA.groupby('st_year')["st_year"].count()
Out[15]:
In [16]:
USA.shape
Out[16]:
In [17]:
# Checking for the Null values.
USA.isna().sum()
Out[17]:
In [18]:
# Replacing NaN value with mode value
USA['Timezone'].fillna(USA['Timezone'].mode()[0], inplace=True)
USA['Temperature(F)'].fillna(USA['Temperature(F)'].mode()[0], inplace=True)
USA['Wind_Chill(F)'].fillna(USA['Wind_Chill(F)'].mode()[0], inplace=True)
USA['Humidity(%)'].fillna(USA['Humidity(%)'].mode()[0], inplace=True)
USA['Pressure(in)'].fillna(USA['Pressure(in)'].mode()[0], inplace=True)
USA['Visibility(mi)'].fillna(USA['Visibility(mi)'].mode()[0], inplace=True)
USA['Wind_Speed(mph)'].fillna(USA['Wind_Speed(mph)'].mode()[0], inplace=True)
USA['Weather_Condition'].fillna(USA['Weather_Condition'].mode()[0], inplace=True)
USA['Sunrise_Sunset'].fillna(USA['Sunrise_Sunset'].mode()[0], inplace=True)
USA['Civil_Twilight'].fillna(USA['Civil_Twilight'].mode()[0], inplace=True)
USA['Nautical_Twilight'].fillna(USA['Nautical_Twilight'].mode()[0], inplace=True)
USA['Astronomical_Twilight'].fillna(USA['Astronomical_Twilight'].mode()[0], inplace=True)
In [19]:
USA.isna().sum()
Out[19]:
EXPLORATORY DATA ANALYSIS
Graphical Representation of Data¶
In [20]:
f,ax=plt.subplots(1,2,figsize=(18,8))
USA['Severity'].value_counts().plot.pie(explode=[0,0.1,0.1,0.1,0.1],autopct='%1.1f%%',ax=ax[0],shadow=True)
ax[0].set_title('Severity Distribution in Percentage')
ax[0].set_ylabel('No. of Accidents')
sns.countplot('Severity',data=USA,ax=ax[1],order=USA['Severity'].value_counts().index)
ax[1].set_title('Count of Accidents Severitywise')
plt.show()
In [21]:
fig, ax=plt.subplots(figsize=(16,7))
USA['Weather_Condition'].value_counts().sort_values(ascending=False).plot.bar(width=0.5,edgecolor='k',align='center',linewidth=2)
plt.xlabel('Weather_Condition',fontsize=20)
plt.ylabel('Number of Accidents',fontsize=20)
ax.tick_params(labelsize=20)
plt.title('No.of accidents based on Weather Condition',fontsize=25)
plt.grid()
plt.ioff()
In [22]:
f,ax=plt.subplots(1,2,figsize=(18,8))
USA['Timezone'].value_counts().plot.pie(explode=[0,0,0.1,0],autopct='%1.1f%%',ax=ax[0],shadow=True)
ax[0].set_title('Accidents in Different Timezone')
sns.countplot('Timezone',data=USA,ax=ax[1],order=USA['Timezone'].value_counts().index)
ax[1].set_title('Accident Count Based on Timezone')
plt.show()
In [23]:
fig, ax=plt.subplots(figsize=(16,10))
USA['State'].value_counts().sort_values(ascending=False).plot.bar(width=0.5,edgecolor='k',align='center',linewidth=2)
plt.xlabel('State',fontsize=20)
plt.ylabel('No. of Accidents',fontsize=20)
ax.tick_params(labelsize=10)
plt.title('Statewise Accidents',fontsize=25)
plt.grid()
plt.ioff()
In [24]:
plt.style.use('ggplot')
fig, ax = plt.subplots(figsize=(15,7))
USA.groupby(['st_year','st_Month','Severity']).count()['State'].unstack().plot(ax=ax)
ax.set_xlabel('Week')
ax.set_ylabel('Number of Accidents')
Out[24]:
Feature Engineering
Next step is to do Feature Engineering, the columns that have data classes need to be dummyfied.¶
In [25]:
USA.head()
Out[25]:
All the below Columns needs to be Dummyfied:¶
- Timezone¶
- Amenity¶
- Bump¶
- Crossing¶
- Give_Way¶
- Junction¶
- No_Exit¶
- Railway¶
- Roundabout¶
- Station¶
- Civil_Twilight¶
- Stop¶
- Traffic_Calming¶
- Traffic_Signal¶
- Turning_Loop¶
- Sunrise_Sunset¶
- Civil_Twilight¶
- Nautical_Twilight¶
- Astronomical_Twilight¶
In [26]:
USA = pd.get_dummies(USA, columns=['Timezone','Amenity',"Bump","Crossing","Give_Way","Junction","No_Exit","Railway","Roundabout",
"Station","Civil_Twilight","Stop","Traffic_Calming","Traffic_Signal","Turning_Loop","Sunrise_Sunset",
"Nautical_Twilight","Astronomical_Twilight"])
In [27]:
USA.columns
Out[27]:
In [28]:
# Dropping this columns now, as there are too many classes, if we change that data into Dummies, then only the number of columns will increase.
USA = USA.drop('Weather_Condition',axis = 1)
In [67]:
USA = USA.drop('Civil_Twilight',axis = 1)
In [30]:
USA.info()
In [31]:
USA.head()
Out[31]:
Linear Regression
Next step is to start with Linear Regression.¶
In [32]:
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import statsmodels.api as sm
In [33]:
USA_Linear = USA
In [34]:
USA_Linear = USA_Linear.drop('Start_Time',axis = 1)
USA_Linear = USA_Linear.drop('End_Time',axis = 1)
USA_Linear = USA_Linear.drop('State',axis = 1)
In [35]:
USA_Linear.shape
Out[35]:
In [36]:
USA_Linear.head()
Out[36]:
In [37]:
USA_Linear.info()
In [38]:
X_train_Li, X_test_Li, Y_train_Li, Y_test_Li = train_test_split(USA_Linear.drop("Severity", axis=1), USA_Linear['Severity'], test_size = 0.2,random_state=112)
In [39]:
my_model = LinearRegression(normalize=True)
In [40]:
my_model.fit(X_train_Li, Y_train_Li)
Out[40]:
In [41]:
predictions = my_model.predict(X_test_Li)
In [42]:
pd.DataFrame({'actual value': Y_test_Li, 'predictions':predictions}).sample(5)
Out[42]:
In [43]:
my_model.score(X_test_Li, Y_test_Li)
Out[43]:
In [44]:
# Checking for Coefficient and Constant Values.
my_model.coef_
Out[44]:
In [45]:
my_model.intercept_
Out[45]:
Finding out the Mean Absolute Error and Mean Sqared Error and R2 Scores.¶
In [46]:
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
In [47]:
mean_absolute_error(Y_test_Li, predictions)
Out[47]:
In [48]:
mean_squared_error(Y_test_Li, predictions)
Out[48]:
In [49]:
r2_score(Y_test_Li, predictions)
Out[49]:
End of Linear Regression.
Logistic Regression
Next step is to start with Logistic Regression.¶
In [50]:
import pandas as pd
import numpy as np
import seaborn as sb
import matplotlib.pyplot as plt
import math
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
import statsmodels.api as sm
In [51]:
# Assigning the USA_Linear dataset only, as there are few columns removed in the USA_Linear data.
USA_Logistic = USA_Linear
In [52]:
USA_Logistic.head()
Out[52]:
In [53]:
USA_Logistic.info()
In [54]:
X_train_Lg, X_test_Lg, Y_train_Lg, Y_test_Lg = train_test_split(USA_Logistic.drop('Severity', axis=1), USA['Severity'],\
test_size=0.2, random_state=156)
In [55]:
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
In [56]:
logit = LogisticRegression()
In [57]:
logit.fit(X_train_Lg, Y_train_Lg)
Out[57]:
In [58]:
predictions = logit.predict(X_test_Lg)
In [117]:
predictions[:5]
Out[117]:
In [59]:
#Calculating Accurary Score:
logit.score(X_test_Lg, Y_test_Lg)
Out[59]:
In [60]:
#Calculating Error:
1-logit.score(X_test_Lg, Y_test_Lg)
Out[60]:
In [61]:
#Calculating Accuracy:
accuracy_score(Y_test_Lg, predictions)
Out[61]:
In [62]:
#Calculate number of correctly classified observations.
accuracy_score(Y_test_Lg, predictions, normalize=False)
Out[62]:
In [63]:
#Calculate number of incorrectly classified observations:
len(Y_test_Lg) - accuracy_score(Y_test_Lg, predictions, normalize=False)
Out[63]:
In [64]:
from sklearn.metrics import log_loss
import numpy as np
In [65]:
#Encode predicted classes and test labels
Y_test_Lg.head()
Out[65]:
In [115]:
log_loss(Y_test_Lg, predictions)
Calculate Confusion Matrix¶
In [68]:
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
In [69]:
confusion_mat = confusion_matrix(Y_test_Lg, predictions)
In [70]:
confusion_df = pd.DataFrame(confusion_mat, index=['A1','A2','A3','A4'], columns=['P1','P2','P3','P4'])
In [71]:
confusion_df
Out[71]:
In [72]:
_=sns.heatmap(confusion_df, cmap='coolwarm', annot=True)
Calculate precision and recall scores¶
In [73]:
from sklearn.metrics import precision_score, recall_score
In [109]:
precision_score(Y_test_Lg, predictions, average=None)
Out[109]:
In [110]:
recall_score(Y_test_Lg, predictions, average=None)
Out[110]:
In [111]:
from sklearn.metrics import f1_score
In [113]:
f1_score(Y_test_Lg, predictions,average=None)
Out[113]:
End of Logistic Regression¶
Decision Tree
Next step is to start with Decision Tree.¶
In [78]:
USA_DT = USA_Linear
In [79]:
USA_DT.columns
Out[79]:
In [80]:
# Importing the train Test package from Sklearn:
from sklearn.model_selection import train_test_split
In [81]:
# Executing the Train Test Split:
X_train_dt, X_test_dt, Y_train_dt, Y_test_dt = train_test_split(USA_DT.drop("Severity", axis=1), USA['Severity'], test_size=0.2, random_state=123)
In [82]:
# Importing Decision Tree Package
from sklearn.tree import DecisionTreeClassifier
In [83]:
tree = DecisionTreeClassifier(max_depth=20)
In [84]:
tree.fit(X_train_dt, Y_train_dt)
Out[84]:
In [85]:
predictions = tree.predict(X_test_dt)
In [86]:
# Checking the top 5 predictions and actual values.
predictions[:5]
Out[86]:
In [87]:
Y_test_dt[:5]
Out[87]:
<font color = 'blue'>The Prediction are more or less accurate, Will evaluate it with Prediction accuracy. </font>¶
In [88]:
# Importing accuracy score and confusion matrix packages.
from sklearn.metrics import accuracy_score, confusion_matrix
In [89]:
accuracy_score(Y_test_dt, predictions)
Out[89]:
In [90]:
accuracy_score(Y_test_dt, predictions, normalize=False)
Out[90]:
<font color = 'blue'>The accuracy score is 67% . </font>¶
In [91]:
confusion_mat_dt = confusion_matrix(Y_test_dt, predictions)
In [92]:
confusion_df = pd.DataFrame(confusion_mat_dt, index=['A1','A2','A3','A4','A5'], columns=['P1','P2','P3','P4','P5'])
In [93]:
confusion_df
Out[93]:
In [94]:
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(Y_test_dt, predictions))
print(classification_report(Y_test_dt, predictions))
In [95]:
# Viewing the Treeout using Graphviz
_=sns.heatmap(confusion_mat, cmap='coolwarm', annot=True)
In [96]:
from sklearn.tree import export_graphviz
import graphviz
In [97]:
dot_data = export_graphviz(tree, filled=True, rounded=True, feature_names=X_train_dt.columns, out_file=None)
In [98]:
graphviz.Source(dot_data)
Out[98]:
In [99]:
import pandas as pd
from pandas import Series
In [100]:
tree.feature_importances_
Out[100]:
In [101]:
pd.Series(tree.feature_importances_, X_train_dt.columns)
Out[101]:
<font color = 'blue'>Pressure, Temparature, Humidity, Traffic_signal_false Predictor has more Importance as compared to other Predictor.</font>¶
In [102]:
from pydotplus import graph_from_dot_data
In [103]:
graph = graph_from_dot_data(dot_data)
In [104]:
graph.write_png("Regressor_tree.png")
Out[104]:
In [106]:
# Need to export the file so that it can be worked upon in the next file:
USA_Linear.to_csv("E:\\Data for Assignment\\US_Accidents_May19_Part_2.csv")
In [118]:
# Import Decision Tree Regressor, and GridSearchCV and fit the model to the training data.
from sklearn.tree import DecisionTreeRegressor
In [119]:
from sklearn.model_selection import GridSearchCV
In [120]:
# Fixing the Max depth and Features
param_grid = [{"max_depth":[3, 4, 5, None], "max_features":[2,3,4]}]
In [121]:
gs = GridSearchCV(estimator=DecisionTreeRegressor(random_state=123),\
param_grid = param_grid,\
cv=20)
In [122]:
# Fitting the Training data
gs.fit(X_train_dt, Y_train_dt)
Out[122]:
In [123]:
# Print out all the hyperparameters combinations that the GridSearchCV has tried.
gs.cv_results_['params']
Out[123]:
In [124]:
# Print best hyperparameters combination.
gs.best_params_
Out[124]:
In [125]:
# Print the rank for all of the tried combinations.
gs.cv_results_['rank_test_score']
Out[125]:
In [126]:
# Print the complete DecisionTree estimator.
gs.best_estimator_
Out[126]:
In [127]:
#Predicting the data
predictions2 = gs.predict(X_test_dt)
In [128]:
# Calculating accuracy of Prediction
from sklearn.metrics import mean_absolute_error, r2_score
In [129]:
mean_absolute_error(Y_test_dt, predictions2)
Out[129]:
In [133]:
r2_score(Y_test_dt, predictions2)
Out[133]: